#encoding: utf-8
# -*- coding: utf-8 -*-
import codecs
import os,sys
from st_common import *

__all__ = ["ne_pipeline"]
 
def LoadResult(work_file):
    fwd=codecs.open(work_file,'r',encoding='gbk')
    #fwd=codecs.open('D:/data/CDT7/test.conll06','r',encoding='UTF8')
    raw=fwd.read()
    fwd.close()
    lines=raw.split('\n')
    
    corpor=[]
    line_vec=[]
    for ln in lines:
        word=ln.split()
            
        words=[ ''.join(wd.split('\n')) for wd in word if len(wd)>0]
        if len(words)==0 and len(line_vec)>0:
            if len(line_vec)>=1:
                corpor.append(line_vec)
            else:
                line_vec=[]
                continue
                #print 'error line_vec:'+'\t'.join(line_vec[0])
                
            line_vec=[]
        else:
            line_vec.append(words)
    return corpor



def ne_pipeline(data_path,source_file,des_file):

    os.chdir('E:\\desktop\\wu-request\\NLPCC 2014 Shared Tasks Guidelines\\crfNER')
    BASE_TAG_PATH  = 'D:\\data\\Shared_Task_NLPCC14\\ParsingAndTag\\'
    
    
    fid = codecs.open(data_path+source_file, 'r', encoding='gbk')
    raw = fid.read()
    fid.close()
    lines = raw.split('\n')
    
    fod = codecs.open(BASE_TAG_PATH+'crfSeg_run\\raw_sens_in.txt','w',encoding = 'gbk')
    for ln in lines:
        for idx in range(0,len(ln)):
            gbk_wd = ln[idx]
            if gbk_wd.strip()!='':
                fod.write(gbk_wd+'\n')
        fod.write('\n')
    fod.close()
    '-----------------------------------------------'
    os.chdir(BASE_TAG_PATH+'crfSeg_run')
    print 'begin segmented'
    os.system('del raw_sens_ou.txt')
    os.system('crfPP_test -m seg_model raw_sens_in.txt >> raw_sens_ou.txt' )
    os.chdir(BASE_TAG_PATH)
    
    seg_rs = LoadResult(BASE_TAG_PATH+'crfSeg_run\\raw_sens_ou.txt')
    segmented_corpus = []
    for ln in seg_rs:
        line = []
        word = ''
        for wdc in ln:
            if wdc[-1]=='B':
                if word:
                    line.append(word)
                word = wdc[0]
            if wdc[-1]=='I':
                word = word+wdc[0]
            if wdc[-1]=='E':
                word = word+wdc[0]
        if word:
            line.append(word)
        segmented_corpus.append(line)
            
    fwd=codecs.open(BASE_TAG_PATH+'crfNER_run1\\raw_sens_pos_in.txt','w',encoding='gbk')
    for ln in segmented_corpus:
        fwd.write('\n'.join(ln))
        fwd.write('\n\n')
    fwd.close()
    '-----------------------------------------------'
    print 'begin tag the word'    
    os.chdir(BASE_TAG_PATH+'crfNER_run1')
    os.system('del raw_sens_pos_ou.txt')
    os.system('crfPP_test -m model_pos raw_sens_pos_in.txt >>raw_sens_pos_ou.txt' )
    os.system('copy %s %s' % (BASE_TAG_PATH+'crfNER_run1\\raw_sens_pos_ou.txt',\
                              BASE_TAG_PATH+'crfNameEntity_run\\raw_sens_ne_in.txt') )
    '-----------------------------------------------'
    print 'begin named entity process'  
    os.chdir(BASE_TAG_PATH+'crfNameEntity_run')
    os.system('del raw_sens_ne.txt')
    os.system('crfPP_test -m ne_model raw_sens_ne_in.txt >>raw_sens_ne.txt' )
    
    ne_rs = LoadResult('raw_sens_ne.txt')
    os.chdir(BASE_TAG_PATH)
    
    named_entity_corpus = []
    for ln in ne_rs:
        line = []
        word = ''
        word_tag = ''
        for wdc in ln:
            tag = wdc[-1]
            if tag[-1]=='O':
                if word:
                    line.append(word+'_'+ word_tag)
                word=''
                word_tag = ''
                line.append(wdc[0]+'_'+wdc[1])
            if tag[-1]=='B':
                if word:
                    line.append(word+'_'+ word_tag)
                word = wdc[0]
                word_tag = tag[0:-2]
            if tag[-1]=='I':
                word = word+wdc[0]
            if tag[-1]=='E':
                word = word+wdc[0]
        if word:
            line.append(word+' '+ word_tag)
        named_entity_corpus.append(line)
        
    fwd=codecs.open(data_path+des_file,'w',encoding='gbk')
    for ln in named_entity_corpus:
        fwd.write('\t'.join(ln))
        fwd.write('\n')
    fwd.close()
    
    print 'finish!'
    
if __name__ == '__main__1':  
    source_file = 'raw_alltext.txt'
    des_file = 'raw_alltext_ne.txt'
    data_path = 'D:\\data\\Shared_Task_NLPCC14\\ParsingAndTag\\'
    ne_pipeline(data_path,source_file,des_file)